import com.gargoylesoftware.htmlunit.*;
import org.apache.commons.logging.LogFactory;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;

import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.PrintStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.logging.Level;

import com.gargoylesoftware.htmlunit.html.HtmlPage;
/**
 * @author shun
 */
public class HttpUtil {
    /**
     * 根据网址获取 Jsoup.Document 文档
     * @param url 网址
     * @return Jsoup.Document 文档，失败返回 null
     */
    public static Document getDocument(String url) {
        WebClient webClient = getClient();
        WebRequest request = null;
        try {
            request = new WebRequest(new URL(url),HttpMethod.GET);
            request.setAdditionalHeader("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:80.0) Gecko/20100101 Firefox/80.0");
        } catch (MalformedURLException e) {
            e.printStackTrace();
        }
        try {
            //模拟浏览器打开一个目标网址
            HtmlPage htmlPage = webClient.getPage(request);
            //为了获取js执行的数据 线程开始沉睡等待
            webClient.waitForBackgroundJavaScript(10000);
            //以xml形式获取响应文本,并转为Document对象return
            String xml = htmlPage.asXml();
            return Jsoup.parse(xml);
        } catch (Exception e) {
            e.printStackTrace();
        }
        return null;
    }

    public static WebClient getClient(){
        WebClient webClient = new WebClient(BrowserVersion.FIREFOX);
        //是否使用不安全的SSL
        webClient.getOptions().setUseInsecureSSL(true);
        //启用JS解释器，默认为true
        webClient.getOptions().setJavaScriptEnabled(true);
        //禁用CSS
        webClient.getOptions().setCssEnabled(false);
        //js运行错误时，是否抛出异常
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        //状态码错误时，是否抛出异常
        webClient.getOptions().setThrowExceptionOnFailingStatusCode(false);
        //是否允许使用ActiveX
        webClient.getOptions().setActiveXNative(true);
        //等待js时间
        webClient.waitForBackgroundJavaScript(600 * 1000);
        //设置Ajax异步处理控制器即启用Ajax支持
        webClient.setAjaxController(new AjaxController());
        //设置超时时间
        webClient.getOptions().setTimeout(1000000);
        //不跟踪抓取
        webClient.getOptions().setDoNotTrackEnabled(false);
        return webClient;
    }
    /**
     * 实现 js escape 功能，将中文字符串转换为 Escape/Unescape %u编码
     * @param str 中文字符串
     * @return %u编码的字符串
     */
    public static String escape(String str) {
        int i;
        char j;
        StringBuffer tmp = new StringBuffer();
        tmp.ensureCapacity(str.length() * 6);
        for (i = 0; i < str.length(); i++) {
            j = str.charAt(i);
            if (Character.isDigit(j) || Character.isLowerCase(j)
                    || Character.isUpperCase(j)) {
                tmp.append(j);
            } else if (j < 256) {
                tmp.append("%");
                if (j < 16) {
                    tmp.append("0");
                }
                tmp.append(Integer.toString(j, 16));
            } else {
                tmp.append("%u");
                tmp.append(Integer.toString(j, 16));
            }
        }
        return tmp.toString();
    }
}
